﻿using MongoDB.Bson;
using MongoDB.Driver;
using Sustainalytics.Ai.HbaseRepository;
using Sustainalytics.BusinessResearchProductStates.Entities.ResearchState;
using Sustainalytics.ClientDW.DataAccessLayer;
using Sustainalytics.ClientDW.Entities;
using Sustainalytics.Entities.ESGRatings;
using Sustainalytics.ESG.Entities;
using Sustainalytics.Events.DataAccessLayer;
using Sustainalytics.MongoDB.Extensions;
using Sustainalytics.Synchronization.Interface;
using Sustainalytics.Utils;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Configuration;
using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;
using EventIndicator = Sustainalytics.ClientDW.Entities.EventIndicator;
using EventIndicatorSummary = Sustainalytics.ClientDW.Entities.EventIndicatorSummary;
using Incident = Sustainalytics.ClientDW.Entities.Incident;

namespace Sustainalytics.ClientDW.Synchronization
{
    public class ControversiesSyncStep : ISyncStep
    {
        public SyncStepOrder ExecutionOrder => SyncStepOrder.SyncControversies;
        private static DateTime? _lastSyncTime;

        public SyncContext Start(SyncContext syncContext)
        {
            syncContext.Log($"Controversies Last Sync Time: {GetLastSyncTime(syncContext)}");

            EnsureConnectionStrings(syncContext);

            EnsureCompanyUniverses(syncContext);

            var eventsDataReader = EventsDataReader.GetInstance(syncContext.GAConnectionStrings["EventsConnectionString"], -1);
            var publishedCompanyProfilesFromEvents = eventsDataReader.GetPublishedCompanyProfiles();
            syncContext.Log(publishedCompanyProfilesFromEvents.Count + " Published Company Profile Ids retrieved.");

            List<Incident> incidents = null;

            var companyIds = publishedCompanyProfilesFromEvents.Select(cp => cp.CompanyId).ToList();

            var hasControversyResearchCompanyIds = EstablishAvailabilityControversyResearchFlags(syncContext, companyIds);

            new Action[]
            {
                () => UpdateCorporateDataLastControversyUpdate(syncContext, publishedCompanyProfilesFromEvents),
                () => SyncEventIndicatorTemplates(syncContext),
                () => incidents = SyncIncidents(syncContext, companyIds),
                () => SyncNewsArticles(syncContext)
            }
            .RunInParallel(syncContext.Token);

            var eventIndicators = SyncEventIndicators(syncContext, companyIds, incidents);

            incidents = null;

            var controversyIndicators = SyncControversyIndicators(syncContext, hasControversyResearchCompanyIds, eventIndicators, GetHbaseConfiguration());

            var controversyIndicatorsDictionary = controversyIndicators.GroupBy(x => x.CompanyId).ToDictionary(x => x.Key, x => x.ToList());
            SyncComparisonGroupControversies(syncContext, controversyIndicatorsDictionary);

            return syncContext;
        }

        private static HbaseConfiguration GetHbaseConfiguration()
        {
            return new HbaseConfiguration
            {
                ThriftServerAddress = ConfigurationManager.AppSettings["HBaseThriftServer"],
                ThriftPort = int.Parse(ConfigurationManager.AppSettings["HBaseThriftPort"]),
                TablePrefix = ConfigurationManager.AppSettings["HBaseTablePrefix"]
            };
        }

        private static void EnsureConnectionStrings(SyncContext syncContext)
        {
            if (!syncContext.GAConnectionStrings.Any())
                throw new ArgumentException("GAConnectionStrings is not provided!");

            if (ConfigurationManager.AppSettings["CommandTimeout"] == null)
                throw new Exception("CommandTimeout key is missing!");

            var commandTimeout = SyncHelper.ReadCommandTimeOutFromConfig();
            syncContext.Log($"DataReader CommandTimeout: {commandTimeout}");

            string connectionString;

            if (!syncContext.GAConnectionStrings.TryGetValue("ClientDW", out connectionString))
            {
                throw new ArgumentException("No connection string found for ClientDW!");
            }

            syncContext.Log("IClientDwRepository acquired based on connection string " + connectionString);

            if (!syncContext.GAConnectionStrings.TryGetValue("RPConnectionString_Temp", out connectionString))
            {
                throw new ArgumentException("No connection string found for RP!");
            }

            syncContext.Log("IRpDataReader acquired based on connection string " + connectionString);

            if (!syncContext.GAConnectionStrings.TryGetValue("EventsConnectionString", out connectionString))
            {
                throw new ArgumentException("No connection string found for Events!");
            }

            syncContext.Log("IEventsDataReader acquired based on connection string " + connectionString);
        }

        private static void EnsureCompanyUniverses(SyncContext syncContext)
        {
            if (syncContext.CompanyUniverses != null) return;

            var universesRepository = UniversesRepository.GetUniversesRepository(syncContext.GAConnectionStrings["ClientUniverses"]);
            syncContext.CompanyUniverses = universesRepository.GetAllUniverseCompany();
        }

        private static DateTime? GetLastSyncTime(SyncContext syncContext)
        {
            var lastSyncTime = syncContext.LastSyncTime;

            var changedSince = SyncHelper.ReadControversiesChangedSinceFromConfig();
            if (changedSince.HasValue)
            {
                lastSyncTime = changedSince == DateTime.MinValue ? null : changedSince;
            }

            var isInitialSync = SyncHelper.ReadIsControversiesInitialSyncFromConfig();

            if (isInitialSync)
                lastSyncTime = null;

            return lastSyncTime;
        }

        private static List<Guid> EstablishAvailabilityControversyResearchFlags(SyncContext syncContext, IEnumerable<Guid> publishedCompanyIds)
        {
            syncContext.Token.ThrowIfCancellationRequested();

            syncContext.Log("Start establish availability of controversy research...");

            var builder = Builders<CompanyProfileResearchState>.Filter;
            var filter = builder.Eq(cp => cp.ResearchParentId, null)
                       & builder.Eq(cp => cp.ControversyResearchState.ResearchState, ResearchState.ResearchAvailable);

            var companyIds =
                MongoFactory.GetCollectionFromDatabase<CompanyProfileResearchState>(syncContext.GAConnectionStrings["ClientDW"])
                    .Find(filter)
                    .Project(cd => cd.Id)
                    .ToList();

            syncContext.Log(companyIds.Count + " companies retrieved with controversy research available.");

            var notAvailableResearchCompanyIds = companyIds.Except(publishedCompanyIds).ToList();

            notAvailableResearchCompanyIds.ForEach(companyId =>
            {
                UpdateCorporateDataToResearchInProgress(syncContext, companyId);
                UpdateCompanyProfileResearchStateToResearchInProgress(syncContext, companyId);
            });

            syncContext.Log(notAvailableResearchCompanyIds.Count + " companies updated with controversy research in progress.");

            return companyIds;
        }

        private static void UpdateCorporateDataToResearchInProgress(SyncContext syncContext, Guid companyId)
        {
            var collection = MongoFactory.GetCollectionFromDatabase<CorporateData>(syncContext.GAConnectionStrings["ClientDW"]);

            var filter = Builders<CorporateData>.Filter.Eq(cd => cd.Id, companyId);
            var update = Builders<CorporateData>.Update
                .Set(cd => cd.ControversyResearchState.ResearchState, ResearchState.ResearchInProgress);

            collection.UpdateOne(filter, update);
        }

        private static void UpdateCompanyProfileResearchStateToResearchInProgress(SyncContext syncContext, Guid companyId)
        {
            var collection = MongoFactory.GetCollectionFromDatabase<CompanyProfileResearchState>(syncContext.GAConnectionStrings["ClientDW"]);

            var filter = Builders<CompanyProfileResearchState>.Filter.Eq(cp => cp.Id, companyId);
            var update = Builders<CompanyProfileResearchState>.Update
                .Set(cd => cd.ControversyResearchState.ResearchState, ResearchState.ResearchInProgress);

            collection.UpdateOne(filter, update);
        }

        private void UpdateCorporateDataLastControversyUpdate(SyncContext syncContext, List<EventIndicatorCompanyProfile> publishedCompanyProfilesFromEvents)
        {
            var sw = Stopwatch.StartNew();

            syncContext.Log(publishedCompanyProfilesFromEvents.Count + " published entities retrieved.");

            var collection = MongoFactory.GetCollectionFromDatabase<CorporateData>(syncContext.GAConnectionStrings["ClientDW"]);

            publishedCompanyProfilesFromEvents.ForEach(item =>
            {
                var filter = Builders<CorporateData>.Filter.Eq(cp => cp.Id, item.CompanyId);
                var update = Builders<CorporateData>.Update
                    .Set(cd => cd.LastControversyUpdate, item.LastUpdated);

                collection.UpdateOne(filter, update);
            });

            sw.Stop();
            syncContext.Log(
                $"Corporate data LastControversyUpdate synchronization completed successfully. Rows affected: {publishedCompanyProfilesFromEvents.Count}, Elapsed time: {sw.Elapsed}");
        }

        private static void SyncNewsArticles(SyncContext syncContext)
        {
            var sw = Stopwatch.StartNew();

            var eventsDataReader = EventsDataReader.GetInstance(syncContext.GAConnectionStrings["EventsConnectionString"], -1);
            var newsArticles = eventsDataReader.GetNewsArticles();
            syncContext.Log(newsArticles.Count + " NewsArticle entities retrieved.");

            var collection = MongoFactory.GetCollectionFromDatabase<NewsArticle>(syncContext.GAConnectionStrings["ClientDW"]);

            SaveDocuments(syncContext, newsArticles, collection);
            CreateIndex(collection).Wait();

            sw.Stop();
            syncContext.Log(
                $"News Articles synchronization completed successfully. Rows affected: {newsArticles.Count}, Elapsed time: {sw.Elapsed}");
        }

        private static void SyncEventIndicatorTemplates(SyncContext syncContext)
        {
            var sw = Stopwatch.StartNew();

            var eventsDataReader = EventsDataReader.GetInstance(syncContext.GAConnectionStrings["EventsConnectionString"], -1);

            var baseEventIndicators = eventsDataReader.ListBaseEventIndicators();
            syncContext.Log(baseEventIndicators.Count + " BaseEventIndicator entities retrieved.");

            if (baseEventIndicators.Any())
            {
                var collection = MongoFactory.GetCollectionFromDatabase<BaseEventIndicator>(syncContext.GAConnectionStrings["ClientDW"]);

                SaveDocuments(syncContext, baseEventIndicators, collection);
                CreateIndex(collection).Wait();
            }

            sw.Stop();
            syncContext.Log($"Base event indicators templates synchronization completed successfully. Rows affected: {baseEventIndicators.Count}, Elapsed time: {sw.Elapsed}");
        }

        private static List<Incident> SyncIncidents(SyncContext syncContext, List<Guid> companyIds)
        {
            var sw = Stopwatch.StartNew();
            var eventsDataReader = EventsDataReader.GetInstance(syncContext.GAConnectionStrings["EventsConnectionString"], -1);

            var incidents = eventsDataReader.GetIncidents(companyIds);
            syncContext.Log(incidents.Count + " Incident entities retrieved.");

            var collection = MongoFactory.GetCollectionFromDatabase<Incident>(syncContext.GAConnectionStrings["ClientDW"]);

            SaveDocuments(syncContext, incidents, collection);
            CreateIndex(collection).Wait();

            sw.Stop();
            syncContext.Log($"Incidents synchronization completed successfully. Rows affected: {incidents.Count}, Elapsed time: {sw.Elapsed}");

            return incidents;
        }

        private static List<EventIndicator> SyncEventIndicators(SyncContext syncContext, List<Guid> companyIds, List<Incident> incidents)
        {
            syncContext.Token.ThrowIfCancellationRequested();

            var sw = Stopwatch.StartNew();

            var eventIndicatorIncidentLocations = new Dictionary<Guid, List<Location>>();
            var eventIndicatorTags = new Dictionary<Guid, List<string>>();

            if (incidents.Any())
            {
                new Action[]
                {
                    () =>
                    {
                        eventIndicatorIncidentLocations = incidents
                            .GroupBy(i => i.EventIndicatorId)
                            .ToDictionary(g => g.Key, g => g.Select(i => new {i.Location.Address, i.Location.Latitude, i.Location.Longitude})
                                                            .Distinct()
                                                            .Select(l => new Location
                                                            {
                                                                Address = l.Address,
                                                                Latitude = l.Latitude,
                                                                Longitude = l.Longitude
                                                            })
                                                            .ToList());
                    },
                    () =>
                    {
                        eventIndicatorTags = incidents.GroupBy(i => i.EventIndicatorId)
                                                        .ToDictionary(g => g.Key, 
                                                                        g => g.Select(i => i.MainTag).Distinct().ToList());
                    }
                }.RunInParallel(syncContext.Token);
            }

            var eventsDataReader = EventsDataReader.GetInstance(syncContext.GAConnectionStrings["EventsConnectionString"], -1);
            var eventIndicators = eventsDataReader.GetEventIndicators(companyIds);
            syncContext.Log(eventIndicators.Count + " EventIndicator entities retrieved.");

            Parallel.ForEach(eventIndicators, new ParallelOptions { MaxDegreeOfParallelism = Convert.ToInt32(Math.Ceiling(Environment.ProcessorCount * 0.75)) },
            ei =>
            {
                ei.Locations = eventIndicatorIncidentLocations.TryGetValue(ei.Id, out List<Location> locations) ? locations : new List<Location>();

                ei.LocationsText = string.Join(", ", ei.Locations.Select(l => l.Address));

                ei.Tags = string.Join(", ", eventIndicatorTags.TryGetValue(ei.Id, out List<string> tags) ? tags : new List<string>());

                ei.UniverseIds = syncContext.CompanyUniverses.TryGetValue(ei.CompanyId, out List<int> universeIds) ? universeIds : null;
            });

            var collection = MongoFactory.GetCollectionFromDatabase<EventIndicator>(syncContext.GAConnectionStrings["ClientDW"]);

            SaveDocuments(syncContext, eventIndicators, collection);
            CreateIndex(collection).Wait();

            sw.Stop();
            syncContext.Log(
                $"Event indicators synchronization completed successfully. Rows affected: {eventIndicators.Count}, Elapsed time: {sw.Elapsed}");

            return eventIndicators;
        }

        private static List<IndicatorTemplate> GetControversyIndicatorTemplates(HbaseConfiguration hbaseConfiguration)
        {
            ResearchDataProvider.ResearchDataProvider researchProvider = new ResearchDataProvider.ResearchDataProvider(hbaseConfiguration);

            var indicatorTemplates = researchProvider.GetIndicatorTemplate(ResearchDataProvider.Models.Analyst_Interface.FrameworkCodes.ComprehensiveRatings).ToList();

            return indicatorTemplates.Where(it => it.IsControversyIndicator)
                                     .Select(x => new IndicatorTemplate
                                     {
                                         Id = x.Id,
                                         IndicatorCategory = x.IndicatorCategory,
                                         IndicatorCategorySortKey = x.IndicatorCategorySortKey,
                                         IsControversyIndicator = x.IsControversyIndicator,
                                         Name = x.Name,
                                         Number = x.Number,
                                         SortKey = x.SortKey,
                                         ThemeId = x.ThemeId,
                                         ThemeName = x.ThemeName,
                                         ThemeSortKey = x.ThemeSortKey
                                     })
                                     .OrderBy(it => it.SortKey)
                                     .ToList();
        }

        private static List<RPControversyIndicator> SyncControversyIndicators(SyncContext syncContext, List<Guid> hasControversyResearchCompanyIds, List<EventIndicator> eventIndicators, HbaseConfiguration hbaseConfiguration)
        {
            syncContext.Token.ThrowIfCancellationRequested();

            var sw = Stopwatch.StartNew();

            var controversyIndicatorTemplates = GetControversyIndicatorTemplates(hbaseConfiguration);

            var controversyIndicators = eventIndicators
                .Where(ei => ei.ProvidesControversy && hasControversyResearchCompanyIds.Contains(ei.CompanyId))
                .Select(ei => new RPControversyIndicator
                {
                    Id = Guid.NewGuid(),
                    CompanyId = ei.CompanyId,
                    Template = controversyIndicatorTemplates
                        .SingleOrDefault(i => i.Number == ei.ControversyIndicatorNumber),
                    AnswerCategory = ei.AnswerCategory,
                    AnswerCategoryScore = ei.AnswerCategoryScore,
                    AnswerCategoryText = ei.AnswerCategoryText,
                    OutlookScore = ei.OutlookScore,
                    Trend = null,
                    Weight = null,
                    WeightedScore = null
                })
                .ToList();

            var eventIndicatorsDictionary = eventIndicators.GroupBy(
                ei => new KeyValuePair<Guid, Guid>(ei.CompanyId, ei.ControversyIndicatorId))
                .ToDictionary(g => g.Key, g => g.Select(ei => new EventIndicatorSummary
                {
                    Id = ei.BaseEventIndicatorId,
                    Name = ei.Name,
                    ControversyIndicatorId = ei.ControversyIndicatorId,
                    ControversyIndicatorName = ei.ControversyIndicatorName,
                    ControversyIndicatorNumber = ei.ControversyIndicatorNumber,
                    AnswerCategory = ei.AnswerCategory
                })
                .ToList());

            var eventIndicatorsConcurrentDictionary = new ConcurrentDictionary<KeyValuePair<Guid, Guid>, List<EventIndicatorSummary>>(eventIndicatorsDictionary);

            var exceptions = new ConcurrentQueue<Exception>();

            Parallel.ForEach(controversyIndicators,
                new ParallelOptions
                {
                    MaxDegreeOfParallelism = Convert.ToInt32(Math.Ceiling(Environment.ProcessorCount * 0.75))
                },
                controversyIndicator =>
                {
                    try
                    {
                        List<EventIndicatorSummary> eventsIndicatorsForCi;

                        if (!eventIndicatorsConcurrentDictionary.TryGetValue(
                            new KeyValuePair<Guid, Guid>(controversyIndicator.CompanyId,
                                controversyIndicator.Template.Id),
                            out eventsIndicatorsForCi))
                        {
                            eventsIndicatorsForCi = new List<EventIndicatorSummary>(0);
                            syncContext.Log($"No events indicators found for controversy indicator with companyId: {controversyIndicator.CompanyId} and template id: {controversyIndicator.Template.Id}");
                        }

                        controversyIndicator.EventIndicators = eventsIndicatorsForCi;

                        List<int> universeIds;

                        controversyIndicator.UniverseIds = syncContext.CompanyUniverses.TryGetValue(controversyIndicator.CompanyId, out universeIds)
                            ? universeIds
                            : null;
                    }
                    catch (Exception ex)
                    {
                        exceptions.Enqueue(ex);
                    }
                });

            if (exceptions.Count > 0)
                throw new AggregateException(exceptions);

            var collection = MongoFactory.GetCollectionFromDatabase<RPControversyIndicator>(syncContext.GAConnectionStrings["ClientDW"]);

            SaveDocuments(syncContext, controversyIndicators, collection);
            CreateIndex(collection).Wait();
            sw.Stop();
            syncContext.Log(
                $"Controversy indicators synchronization completed successfully. Rows affected: {controversyIndicators.Count}, Elapsed time: {sw.Elapsed}");

            return controversyIndicators;
        }

        private void SyncComparisonGroupControversies(SyncContext syncContext, Dictionary<Guid, List<RPControversyIndicator>> controversyIndicators)
        {
            syncContext.Log("Begin sync comparison group controversies");

            var sw = Stopwatch.StartNew();

            var defaultEsgProfile = new ESGProfile()
            {
                ComparativeGroup = ComparativeGroupEnum.PeerGroup,
                CompanyType = CompanyTypeEnum.Combined,
                TemplateType = TemplateTypeEnum.Combined
            };

            var changedElement = syncContext.CurrentChanges.FirstOrDefault(
                a => a.AffectedObjectName == ChangedElement.ESG_CONTROVERSY_PROFILE);

            if (changedElement != null && changedElement.Detail.Any(a => a.Key.Equals("ESGProfile")))
            {
                var profile = changedElement.Detail["ESGProfile"] as ESGProfile;
                if (profile != null)
                {
                    defaultEsgProfile = profile;
                }
            }

            var companyProfiles = GetCompanyProfiles(syncContext);
            var comparisonGroupDictionary = GetGroupCompanies(companyProfiles, defaultEsgProfile);
            var comparisonGroupControversies = new ConcurrentBag<ComparisonGroupControversy>();

            Parallel.ForEach(companyProfiles,
                new ParallelOptions
                {
                    MaxDegreeOfParallelism = Convert.ToInt32(Math.Ceiling(Environment.ProcessorCount * 0.75))
                },
                companyProfile =>
                {
                    if (string.IsNullOrEmpty(companyProfile.PeerGroup) ||
                        string.IsNullOrEmpty(companyProfile.SustainalyticsIndustryGroup)) return;

                    var comparativeGroup = defaultEsgProfile.ComparativeGroup == ComparativeGroupEnum.PeerGroup
                        ? companyProfile.PeerGroup
                        : companyProfile.SustainalyticsIndustryGroup;

                    var companyIds = comparisonGroupDictionary.ContainsKey(comparativeGroup)
                        ? comparisonGroupDictionary[comparativeGroup]
                        : null;

                    if (companyIds == null) return;

                    var comparativeGroupControversyIndicators = new List<RPControversyIndicator>();
                    companyIds.ForEach(companyId =>
                    {
                        List<RPControversyIndicator> memoryControversyIndicators;

                        if (controversyIndicators.TryGetValue(companyId, out memoryControversyIndicators))
                            comparativeGroupControversyIndicators.AddRange(memoryControversyIndicators);
                    });

                    if (!comparativeGroupControversyIndicators.Any()) return;

                    var controversies = comparativeGroupControversyIndicators
                        .GroupBy(ci => new { ci.AnswerCategory, ci.Template.Number, ci.Template.SortKey })
                        .Select(group => new
                        {
                            Category = group.Key.AnswerCategory,
                            Count = group.Count(),
                            ControversyIndicatorNumber = group.Key.Number,
                            ControversyIndicatorSortKey = group.Key.SortKey
                        })
                        .OrderBy(c => c.Category)
                        .ThenBy(c => c.ControversyIndicatorSortKey)
                        .ToList();

                    var cgcs = controversies.Select(controvery => new ComparisonGroupControversy()
                    {
                        ControversyIndicatorNumber = controvery.ControversyIndicatorNumber,
                        Category = controvery.Category,
                        Percent = Convert.ToInt32((controvery.Count / (decimal)companyIds.Count) * 100),
                        Id = Guid.NewGuid(),
                        CompanyId = companyProfile.CompanyId
                    })
                    .ToList();

                    cgcs.ForEach(cgc =>
                    {
                        cgc.Id = Guid.NewGuid();
                        cgc.CompanyId = companyProfile.CompanyId;
                        comparisonGroupControversies.Add(cgc);
                    });
                });

            var collection = MongoFactory.GetCollectionFromDatabase<ComparisonGroupControversy>(syncContext.GAConnectionStrings["ClientDW"]);

            SaveDocuments(syncContext, comparisonGroupControversies.ToList(), collection);
            CreateIndex(collection).Wait();
            sw.Stop();

            syncContext.Log(
                $"Comparison group controversies completed successfully. Rows affected: {comparisonGroupControversies.Count}, Elapsed time: {sw.Elapsed}");
        }

        public List<CompanyProfile> GetCompanyProfiles(SyncContext syncContext)
        {
            return MongoFactory.GetCollectionFromDatabase<CorporateData>(syncContext.GAConnectionStrings["ClientDW"])
                .Find(_ => _.ControversyResearchState.ResearchState == ResearchState.ResearchAvailable)
                .Project(cd => new
                {
                    cd.Id,
                    cd.CompanyName,
                    cd.CompanyType,
                    cd.TemplateType,
                    cd.PeerGroup,
                    cd.SustainalyticsIndustryGroup
                })
                .ToEnumerable()
                .Select(cd => new CompanyProfile
                {
                    CompanyId = cd.Id,
                    CompanyName = cd.CompanyName,
                    CompanyType = cd.CompanyType,
                    TemplateType = cd.TemplateType,
                    PeerGroup = cd.PeerGroup,
                    SustainalyticsIndustryGroup = cd.SustainalyticsIndustryGroup
                })
                .ToList();
        }

        private static Dictionary<string, List<Guid>> GetGroupCompanies(IEnumerable<CompanyProfile> companyProfiles, ESGProfile esgProfile)
        {
            var profiles = companyProfiles.AsQueryable();

            if (esgProfile.CompanyType == CompanyTypeEnum.Public)
                profiles = profiles.Where(c => c.CompanyType == "Public");

            if (esgProfile.CompanyType == CompanyTypeEnum.Private)
                profiles = profiles.Where(c => c.CompanyType == "Private");

            if (esgProfile.TemplateType == TemplateTypeEnum.TypeA)
                profiles = profiles.Where(c => c.TemplateType == "Type A");

            if (esgProfile.TemplateType == TemplateTypeEnum.TypeB)
                profiles = profiles.Where(c => c.TemplateType == "Type B");

            Dictionary<string, List<Guid>> groupCompanies;
            if (esgProfile.ComparativeGroup == ComparativeGroupEnum.PeerGroup)
            {
                groupCompanies = profiles.Where(a => !string.IsNullOrEmpty(a.PeerGroup)).GroupBy(p => p.PeerGroup)
                    .ToDictionary(g => g.Key, g => g.Select(c => c.CompanyId).ToList());
            }
            else
            {
                groupCompanies = profiles.Where(a => !string.IsNullOrEmpty(a.SustainalyticsIndustryGroup)).GroupBy(p => p.SustainalyticsIndustryGroup)
                    .ToDictionary(g => g.Key, g => g.Select(c => c.CompanyId).ToList());
            }
            return groupCompanies;
        }

        private static void SaveDocuments<T>(SyncContext syncContext, List<T> documents, IMongoCollection<T> collection)
        {
            if (documents == null || documents.Count == 0)
                throw new ArgumentException($"A list of {typeof(T).Name} documents must be provided!");

            collection.Database.DropCollection(typeof(T).Name);

            syncContext.Log($"Collection {typeof(T).Name} was dropped.");

            Parallel.ForEach(documents.Batch(10000), documentsBatch => collection.InsertMany(documentsBatch));

            var rowsAffected = collection.Count(new BsonDocument());

            syncContext.Log($"{rowsAffected} {typeof(T).Name} inserted.");

            if (documents.Count != rowsAffected)
                throw new Exception($"Insert {typeof(T).Name} documents failed!");
        }

        private static async Task CreateIndex(IMongoCollection<Incident> collection)
        {
            await collection.Indexes.CreateOneAsync(Builders<Incident>.IndexKeys.Ascending(x => x.Id)).ConfigureAwait(false);
            await collection.Indexes.CreateOneAsync(Builders<Incident>.IndexKeys.Ascending(x => x.CompanyId)).ConfigureAwait(false);
            await collection.Indexes.CreateOneAsync(Builders<Incident>.IndexKeys.Ascending(x => x.EventIndicatorId)).ConfigureAwait(false);
            await collection.Indexes.CreateOneAsync(Builders<Incident>.IndexKeys.Ascending(x => x.IncidentId)).ConfigureAwait(false);
        }

        private static async Task CreateIndex(IMongoCollection<RPControversyIndicator> collection)
        {
            await collection.Indexes.CreateOneAsync(Builders<RPControversyIndicator>.IndexKeys.Ascending(x => x.Id)).ConfigureAwait(false);
            await collection.Indexes.CreateOneAsync(Builders<RPControversyIndicator>.IndexKeys.Ascending(x => x.CompanyId)).ConfigureAwait(false);
        }

        private static async Task CreateIndex(IMongoCollection<NewsArticle> collection)
        {
            await collection.Indexes.CreateOneAsync(Builders<NewsArticle>.IndexKeys.Ascending(x => x.Id)).ConfigureAwait(false);
            await collection.Indexes.CreateOneAsync(Builders<NewsArticle>.IndexKeys.Ascending(x => x.CompanyId)).ConfigureAwait(false);
        }

        private static async Task CreateIndex(IMongoCollection<EventIndicator> collection)
        {
            await collection.Indexes.CreateOneAsync(Builders<EventIndicator>.IndexKeys.Ascending(x => x.Id)).ConfigureAwait(false);
            await collection.Indexes.CreateOneAsync(Builders<EventIndicator>.IndexKeys.Ascending(x => x.CompanyId)).ConfigureAwait(false);
            await collection.Indexes.CreateOneAsync(Builders<EventIndicator>.IndexKeys.Ascending(x => x.BaseEventIndicatorId)).ConfigureAwait(false);
            await collection.Indexes.CreateOneAsync(Builders<EventIndicator>.IndexKeys.Ascending(x => x.AnswerCategory)).ConfigureAwait(false);
            await collection.Indexes.CreateOneAsync(Builders<EventIndicator>.IndexKeys.Ascending(x => x.CompanyId).Ascending(x => x.BaseEventIndicatorId)).ConfigureAwait(false);
        }

        private static async Task CreateIndex(IMongoCollection<BaseEventIndicator> collection)
        {
            await collection.Indexes.CreateOneAsync(Builders<BaseEventIndicator>.IndexKeys.Ascending(x => x.Id)).ConfigureAwait(false);
        }

        private static async Task CreateIndex(IMongoCollection<ComparisonGroupControversy> collection)
        {
            await collection.Indexes.CreateOneAsync(Builders<ComparisonGroupControversy>.IndexKeys.Ascending(x => x.Id)).ConfigureAwait(false);
            await collection.Indexes.CreateOneAsync(Builders<ComparisonGroupControversy>.IndexKeys.Ascending(x => x.CompanyId)).ConfigureAwait(false);
        }
    }
}
